#!/usr/bin/python

from xml.sax.saxutils import escape
from threading import Semaphore, Thread
import sys, traceback, re
from StringIO import StringIO

#from local
from lib.database_qqshuqian import Database, tt_init, tt_close, tt_get, tt_put, tt_keys
from lib.html_extractor import extract_text
from lib.utils import sanitizestring, transcode5
from lib.network import clean_url, fetch_url3


workers_semaphore = Semaphore(8)

def redirect_filter(txt):
    if not txt:
	return 
    try:
        match = re.findall(ur'\u8bf7\u70b9\u6b64\u8fdb\u5165\uff01', txt.decode('utf-8'))
    except:
	return
#    print 'lllllllll',len(match)
    if len(match) >= 3:
	return True
    else:
	return False

def make_worker(linkid, linkhtml):
    def worker():
        global workers_semaphore
        tcinstance = tt_init()
        #tcinstance_txt = tt_init('192.168.1.103')
        tcinstance_txt = tt_init('localhost')
#	filename = linkid+".html"
#    	f = open(filename,'w')
        try:
            if linkhtml:
                html = transcode5(linkhtml)
                if html:
                    txt = extract_text(html, use_ann=True)
#		    print >> f,html
#		    print "@@@@",txt
                    if txt:
		        if redirect_filter(txt):
#			    print 'hhhhhhhhhaaaaaaaaaa'
			    return
		        if len(txt) < 20:
			    return 
                        tt_put(tcinstance_txt, "txt_" + linkid, txt)
        except:
            txt = ''
            print >>sys.stderr, ('ERROR in extract_text() with the content from linkid %s\n' % linkid)
            sio = StringIO()
            traceback.print_exc(file=sio)
            print sio.getvalue()
            sio.close()
        tt_close(tcinstance_txt)
        tt_close(tcinstance)
        workers_semaphore.release()
    return worker


def processHTML():
    global workers_semaphore
    tcinstance = tt_init()
    #tcinstance_txt = tt_init('192.168.1.103')
    tcinstance_txt = tt_init('localhost')
    i = 0
    tt_put(tcinstance_txt, "EMPTY", "EMPTY")
    for key in tt_keys(tcinstance):
        if key.startswith('date_') or key.startswith('txt_'):
            pass
        else:
            if tt_get(tcinstance_txt, "txt_" + key) == None:
                print 'working on ', i, key
                linkhtml = tt_get(tcinstance, key)
                if linkhtml:
                    worker_thread = Thread(target = make_worker(key, linkhtml))
#		    wait = raw_input("Press Enter to continue..")
                    workers_semaphore.acquire()
                    worker_thread.start()
                i += 1
    tt_close(tcinstance)
    tt_close(tcinstance_txt)

processHTML()
